package engine

import (
	"gitee.com/jnshao/go-start/projects/crawler/fetcher"
	"log"
)

type SimpleEngine struct{}

func (engine SimpleEngine) Run(seeds ...Request) {
	requests := []Request{}
	requests = append(requests, seeds...)

	for len(requests) > 0 {
		request := requests[0]
		requests = requests[1:]

		requestResult, err := Worker(request)
		if err != nil {
			continue
		}

		requests = append(requests, requestResult.Requests...)
	}

}

func Worker(request Request) (ParserResult, error) {
	bytes, err := fetcher.Fetch(string(request.URL))
	if err != nil {
		log.Printf("Fetcher Error:URL:%s,ERR:%v", request.URL, err)
		return ParserResult{}, err
	}
	log.Printf("Fetcher:URL:%s\n", request.URL)
	return request.Parser.Parse(bytes), nil
}
